var citiesList = "sh";

//var cities = citiesList.split(",");
var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/Communities';
var async = require('async');

var MongoClient = require('mongodb').MongoClient, assert = require('assert');

var dbconn = null;

cities.forEach(function(city){
//async.eachSeries(cities, function (city) {

    var url = 'http://' + city + '.sofang.com/ershoufang?cityid=&cityareaid=&busid=&pricetype=&housetype=&keywords=&page=';
    var dbCollection = 'xq_sf_' + city;
    var totalPages = 1;
    var totalComms = 0;
    var counter = 0;
    var numPerPage = 9;
    var db = null;

    function getTotalPages(callback) {
        // Parse Total Pages
        request(url + '1', function (err, resp, body) {
            if (!err && resp.statusCode == 200) {
                var $ = cheerio.load(body);
                var li_pages = $("div.page").find("ul").children();
                var plen = li_pages.length
                for(var i = 0; i < plen; i++){
                    if(li_pages[i].children.length > 0){
                        var obj = li_pages[i].children[0]
                        if(obj.name == 'a'){
                            var subobj = obj.children[0]
                            if (subobj != undefined && "data" in subobj){
                                var pageNo = parseInt(subobj.data)
                                if (pageNo > totalPages)
                                    totalPages = pageNo
                            }
                        }
                    }
                }
                totalComms = numPerPage * totalPages
                console.log("Total Pages for " + city.toUpperCase() + ": " + totalPages);
                console.log("Upper limit total Comms for " + city.toUpperCase() + ": " + totalComms);
            }
            callback();
        });
    };

    function run() {

        var manifest = {
            target: {
                url: url + '{{number}}',
                ranges: {
                    number: [1, totalPages, 1]
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            //plugins: {
            //    mongodb: {
            //        connection: connection
            //    },
            //    csv: true
            //},

            extractors: [{
                name: '66xiaoqu',
                root: '//*[@id="list"]',
                fields: {
                    //name: '/tbody/tr[1]',
                    //href: '//div[@class="details"]/div[@class="t_b"]/a[@class="t"]/@href',
                    //addr: '//div[@class="details"]/p[1]/text()',
                    //fdate: '//div[@class="details"]/p[2]/text()',
                    //map: '//div[@class="details"]/p[@class="p_links"]/a[1]/@href',
                    //price: '//span[@class="price"]/span["sp1"]/text()',
                }
            }
            ]
        };

        function insertDocuments(doc) {

            var collection = dbconn.collection(dbCollection)
            doc['source'] = 'sofang'

            collection.update({"commId":doc.commid},doc, {"upsert":true},function (err, result) {
                counter ++
                //assert.equal(err, null);
                //assert.equal(1, result.result.n);
                //assert.equal(1, result.ops.length);
                console.log("Inserted %s into %s (%s of %s(upper bound))", doc.name, dbCollection, counter, totalComms);
                //callback(result);
            });
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
            console.log(data.request.url)
            var $ = cheerio.load(data.request.body);
            var rows = $('div.list_msg').find('tr')
            var rlen = rows.length
            for(var i = 0; i < rlen; i++){
                if(rows[i].children.length > 1){

                    var pricedataStr = rows[i].children[1].children[3].children[0].data
                    var pricedata = pricedataStr.substring(pricedataStr.indexOf(",'")+2,pricedataStr.indexOf("')")).split(',')
                    var housedata = rows[i].children[3].children[1].children

                    var doc = {}

                    doc.name = housedata[1].children[0].children[0].data
                    doc.href = housedata[1].children[0].attribs.href
                    doc.type = housedata[1].children[1].children[0].data
                    doc.addr = housedata[3].children[0].data
                    doc.year = parseInt(housedata[5].children[2].data.replace(/[^0-9\.]+/g, ''))

                    var commidStr = doc.href.split('/')[2]
                    doc.commid = commidStr.substring(0, commidStr.indexOf('-1.htm'))

                    var pdLen = pricedata.length
                    if(pdLen > 0){
                        // parsing price data
                        doc.priceUpper = 0
                        doc.priceLower = 0
                        doc.priceLatest = 0

                        for(var j =0;j < pdLen; j++){
                            var priceT = parseInt(pricedata[j])
                            doc.priceLatest = priceT
                            if(j == 0){
                                doc.priceLower = priceT
                            }
                            if (priceT > doc.priceUpper)
                                doc.priceUpper = priceT
                            else if (priceT < doc.priceLower)
                                doc.priceLower = priceT
                        }
                    }
                    //console.log(doc)
                    insertDocuments(doc);
                }
            }
            //data.data.forEach(
            //    function (doc) {
            //        //console.log(doc)
            //        //insertDocuments(doc);
            //    })
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            console.log('The job is done');
            db.close()
            // not exactly,
        });

        function init(callback) {
            // Connect to MongoDB
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    //collection = dbconn.collection(dbCollection)
                    callback(null, 'init');
                });
        }

        function start(callback) {
            console.log("starting...");
            job.start();
            callback(null, 'start');
        }

        function finish(callback) {
            console.log("closing db...");
//    dbconn.close();
            callback(null, 'end');
        }


// Start our job
        async.series([
                //getTotalPages,
                init,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);
                console.log('END');
            });
    }

    // Main method
    getTotalPages(run);
    //run()
});

/*

 node /workspace/CrawlerJS/examples/cities/sofang-master.js


 fy fz fuzhou fs fcg fushun fx ganzhou gz gl guigang guangyuan ga ganzi gy gnzz guoluo guyuan

 hd hn huaibei huangshan hs heze hh huizhou heyuan hezhou hc haikou hu honghe hanzhong hlbe haidong haibei huangnan hnzz hx hm ht hld hrb hegang heihe ha hz huzhou hf

 jdz jj ja jn jining jiaozuo jingmen jingzhou jincheng jz jm jy jyg jinchang jq jiangyin jinzhou jl jixi jms jx jh

 kaifeng km klmy kzls ksd kunshan

 la lf ly lw linyi lc luoyang luohe ld liuzhou linfen lb luzhou lvliang ls liangshan lps lj lincang lasa linzhi lz ln lx liaoyang liaoyuan lyg lishui



 mas mm mz my ms mdj np nc ny nn neijiang nanchong nujiang nqd ningde nj nt nb pt px pds puyang panzhihua pl pj

 qhd qz qd qingyuan qinzhou qxn qdn qn qj qingyang qqhr qth quzhou rizhao rkzd

 sjz suzhou sm sr smx sq shiyan suizhou shaoyang shuozhou sg sz st sw sanya suining simao shannandi sl szs shz sy sp songyuan suihua su suqian sx

 tj ts tongling ty ta tr tongliao tc tianshui tlfd tcd tw taicang tl th taizhou tz

 wuhu wf weihai wh wuzhou wuhai ws wn wuwei wuzhong wlmq wlcb wlcbm wx wz wenchang

 xt xuancheng xm xinyu xx xc xy xiaogan xianning xiangtan xiangxi xinzhou xa xianyang xn xam xg xiangyang xl xsbn xz

 yingtan yichun yt yq yc yy yiyang yongzhou yuncheng yj yf yulin yb ya yx yanan yl ys yinchuan yili yiwu yk yanbian yashan yich yancheng yz

 zjk zhangzhou zb zaozhuang zz zk zmd zhuzhou zjj zh zhanjiang za zs zg zy zunyi zt zhangye zw zjw zj zhoushan
 */